import json
import random
import time

from lxml import etree
from selenium import webdriver
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from xpinyin import Pinyin

url = 'https://so.m.jd.com/webportal/channel/m_category?searchFrom=home'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 '
                  'Safari/537.36 '
}

driver = webdriver.Chrome()
driver.get("https://so.m.jd.com/webportal/channel/m_category?searchFrom=home")
wait = WebDriverWait(driver, 50)

# 保存商品标签
product_tags = []
# 保存商品轮播图片
product_images = []
# 保存商品信息
product_info = []

# 一级分类类别, 从0开始
cate_1 = [9]
# 二级分类类别
cate_2 = [1, 2, 3, 4]
# 三级分类类别
cate_3 = range(2, 7)


def spider(driver):
    wait.until(
        EC.presence_of_element_located((By.XPATH, '//*[@id="category2"]//li'))
    )
    category_1_html = etree.HTML(driver.page_source)
    p = Pinyin()
    category_list = []
    category_1_lis = category_1_html.xpath('//*[@id="category2"]//li')
    for category_1_index, category_1_li in enumerate(category_1_lis):
        # 筛选自己需要的一级分类
        if category_1_index in cate_1:
            print('****进入第%s个一级分类****' % (category_1_index + 1))
            category_1_name = category_1_li.xpath('.//text()')[0]
            category_1_code = p.get_pinyin(category_1_name, '')
            category_1_level = 1
            category_1_sub = []
            # 点击分类跳转并定位到对应的内容
            category_1_button = driver.find_element_by_xpath('//*[@id="category%s"]/a' % str(category_1_index + 6))
            driver.execute_script("arguments[0].click();", category_1_button)
            driver.execute_script('window.scrollBy(0, document.body.scrollHeight)')
            time.sleep(4)
            branch_html = etree.HTML(driver.page_source)
            branch_list = branch_html.xpath('//*[@id="branchList"]/div')
            for branch_list_index, branch_list_div in enumerate(branch_list):
                # 筛选自己需要的二级分类
                if branch_list_index in cate_2:
                    print('****进入分类%s-%s****' % ((category_1_index + 1), (branch_list_index + 1)))
                    category_2_name = (branch_list_div.xpath('./h4/text()'))[0]
                    category_2_code = p.get_pinyin(category_2_name, '')
                    category_2_level = 2
                    category_2_sub = []
                    category_3_lis = branch_list_div.xpath('./ul/li')
                    for category_3_index, category_3_li in enumerate(category_3_lis):
                        # 筛选自己需要的三级分类
                        if category_3_index in cate_3:
                            print('****进入分类%s-%s-%s****' % (
                                (category_1_index + 1), (branch_list_index + 1), (category_3_index + 1)))
                            category_3_name = (category_3_li.xpath('.//span/text()'))[0]
                            category_3_code = p.get_pinyin(category_3_name, '')
                            category_3_img = category_3_li.xpath('./a/img/@src')[0]
                            category_3_level = 3
                            category_3_href = category_3_li.xpath('./a/@href')[0]
                            # 跳转到三级分类列表页，另开窗口
                            driver.execute_script("window.open ('%s')" % category_3_href)
                            driver.switch_to.window(driver.window_handles[1])
                            time.sleep(4)
                            # 判断是否会存在五秒倒计时对话框
                            flag = isElementExist(driver, '//*[@id="pcprompt-viewpc"]')
                            if flag:
                                start = driver.find_element_by_xpath('//*[@id="pcprompt-viewpc"]')
                                start.click()
                            print('****开始爬取%s-%s-%s分类商品详细信息****' % (
                                (category_1_index + 1), (branch_list_index + 1), (category_3_index + 1)))
                            getGoodList(driver, category_3_name)
                            print("****爬取%s-%s-%s分类下的商品信息完成****" % (
                                (category_1_index + 1), (branch_list_index + 1), (category_3_index + 1)))
                            driver.close()
                            driver.switch_to.window(driver.window_handles[0])
                            category_3 = {
                                "category_code": category_3_code,
                                "category_name": category_3_name,
                                "category_level": category_3_level,
                                'category_img': category_3_img
                            }
                            category_2_sub.append(category_3)
                    category_2 = {
                        "category_code": category_2_code,
                        "category_name": category_2_name,
                        "category_level": category_2_level,
                        "category_sub": category_2_sub
                    }
                    category_1_sub.append(category_2)
            category_1 = {
                "category_code": category_1_code,
                "category_name": category_1_name,
                "category_level": category_1_level,
                "category_sub": category_1_sub
            }
            category_list.append(category_1)
            with open('../json/4/categoryList.json', 'w', encoding='utf-8') as fp:
                json.dump(category_list, fp, ensure_ascii=False)


# 判断某元素是否存在
def isElementExist(driver, path):
    flag = True
    try:
        driver.find_element_by_xpath(path)
        return flag
    except:
        flag = False
        return flag


# 根据图片地址截取图片名称划取img的id（主键）
def splitImgId(src):
    img_id = ''
    img_list = src.split('/')
    for i, li in enumerate(img_list):
        if li.find('.jpg', 0, len(li)) != -1 or li.find('.png', 0, len(li)) != -1:
            for key in li:
                if key == '.':
                    break
                img_id = img_id + key
    return img_id


def getGoodList(driver, parent):
    driver.execute_script('window.scrollBy(0, document.body.scrollHeight)')
    time.sleep(4)
    wait.until(
        EC.presence_of_element_located((By.XPATH, "//*[@id='itemList']/div[@class='search_prolist_item']//div["
                                                  "@class='search_prolist_cover']//img"))
    )
    product_list_page = etree.HTML(driver.page_source)
    product_list = product_list_page.xpath("//*[@id='itemList']/div[@class='search_prolist_item']")
    for product_list_item_index, product_list_item in enumerate(product_list):
        if product_list_item_index < 10:
            try:
                good_sn = product_list_item.xpath('./@skuid')[0]
            except:
                good_sn = 0
            try:
                good_name = product_list_item.xpath(".//div[@class='search_prolist_title']/text()")[0].strip()
            except:
                good_name = ""
            try:
                shop_price = product_list_item.xpath(".//div[@class='search_prolist_price']//span/text()")[0].strip()
                market_price = round(float(shop_price) * 1.15, 1)
            except:
                shop_price = 0
                market_price = 0
            try:
                good_front_image = product_list_item.xpath(".//img[@class='photo']/@src")[0]
            except:
                good_front_image = ''
            # 保存商品tag
            good_tags = product_list_item.xpath(".//div[@class='search_prolist_attr']/span/text()")
            for tag in good_tags:
                temp = {
                    'good_sn': good_sn,
                    'good_tag': tag.strip()
                }
                product_tags.append(temp)
            # 打开商品详情页
            good_detail_url = "https://item.m.jd.com/product/%s.html?m2wq_ismiao=1" % good_sn
            driver.execute_script("window.open ('%s')" % good_detail_url)
            driver.switch_to.window(driver.window_handles[2])
            # 保存一条商品的时间间隔，防止因速度太快被禁止访问
            time.sleep(3)
            wait.until(
                EC.presence_of_element_located((By.XPATH, '//*[@id="loopImgUl"]/li'))
            )
            # 获取商品详情页的html
            detail_html = etree.HTML(driver.page_source)
            # 保存商品图片
            good_images = detail_html.xpath('//*[@id="loopImgUl"]/li')
            for good_images_index, good_image in enumerate(good_images):
                if good_images_index == 0:
                    image_url = good_image.xpath('./img/@src')[0]
                else:
                    image_url = good_image.xpath('./img/@back_src')[0]
                image_id = splitImgId(image_url)
                image = {
                    'image_id': image_id,
                    'good_sn': good_sn,
                    'image_url': image_url
                }
                product_images.append(image)
            try:
                good_brief = detail_html.xpath('//*[@id="itemDesc"]/text()')[0].strip()
            except:
                good_brief = ""
            product = {
                'good_sn': good_sn,
                'good_name': good_name,
                'good_front_image': good_front_image,
                'shop_price': float(shop_price),
                'stock_num': random.randint(200, 1000),
                'market_price': market_price,
                'good_brief': good_brief,
                'category_name': parent
            }
            product_info.append(product)
            driver.close()
            driver.switch_to.window(driver.window_handles[1])


def main():
    spider(driver)
    with open('../json/4/commodity.json', 'w', encoding='utf-8') as fp:
        json.dump(product_info, fp, ensure_ascii=False)
    with open('../json/4/tags.json', 'w', encoding='utf-8') as fp:
        json.dump(product_tags, fp, ensure_ascii=False)
    with open('../json/4/images.json', 'w', encoding='utf-8') as fp:
        json.dump(product_images, fp, ensure_ascii=False)


if __name__ == '__main__':
    main()
